R/DB Update.R

In william-rinauto/fdicDB: Build and Query FDIC Data

#Update db

fdicUpdate <- function(dbName){

  #W hat currently exists in db
  db <- DBI::dbConnect(RSQLite::SQLite(), dbName)
  tables <- DBI::dbListTables(db)

  #What is available on website

  page <- rvest::html('https://www5.fdic.gov/sdi/download_large_list_outside.asp')
  onClickURL <- 'https://www5.fdic.gov/sdi/'

  downloads <- page %>%
    rvest::html_nodes('a') %>%
    stringr::str_subset('\\.zip')

  downloads1 <- stringr::str_match(downloads,
                                   '<a href=\\\"(.+\\d{8}\\.zip)\\\">(All_Reports_\\d{8})\\.zip<\\/a>')

  downloads1 <- na.omit(downloads1)

  alreadyWritten <- unique(stringr::str_match(tables, "(.+\\d{8}).+")[,2])

  #Only show quarters that haven't been written to DB yet.
  downloads1 <- subset(downloads1,  !(downloads1[,3] %in% alreadyWritten))


  #downloads1 <- downloads1[1:2,]

  if(!dir.exists(zips)) dir.create(zips)

  lapply(1:nrow(downloads1),function(i){
    download.file(paste(onClickURL,downloads1[i,2],
                        sep = ""),
                  file.path(zips,paste0(downloads1[i,3],".zip")))
  })

  writeFun(dbName, downloads1[,2])
}